# -*- coding: utf-8 -*-

import scrapy
from urllib.parse import urljoin, urlparse
from scrapy import Request
from bs4 import BeautifulSoup
from .fire_and_life_safety import FireAndLifeSafetyEcosystemParser
from .news_detail import NewsDetailParser
from .why_nfpa_code_and_sm import WhyNfpaCodeAndSMParser
from .code_and_standard_list import CodeAndStandardListParser
from .cate_list_in_building_and_life import CateListInBuildingAndLifeParser
from .selenium_utils import get_selenium_renderer, close_selenium_renderer
from fire_control_spider.config import DEBUG



class NFPASiteSpider(scrapy.Spider):
    """NFPA站点主爬虫 - 使用模块化页面解析器"""
    
    name = 'nfpa'
    allowed_domains = ['nfpa.org', 'www.nfpa.org']
    
    base_url = "https://www.nfpa.org"
    

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        
        # 初始化页面解析器
        self.parsers = [
            FireAndLifeSafetyEcosystemParser(self),
            NewsDetailParser(self),
            WhyNfpaCodeAndSMParser(self),
            CodeAndStandardListParser(self),
            CateListInBuildingAndLifeParser(self),
        ]
        self.route()
        
        # 初始化 Selenium 渲染器
        self.selenium_renderer = get_selenium_renderer()

    def route(self):
        """设置起始URL"""
        self.start_urls = []

        # 添加起始URL
        start_urls = [
            'https://www.nfpa.org/en/about-nfpa/nfpa-fire-and-life-safety-ecosystem',
            'https://www.nfpa.org/education-and-research/Research/nfpa-research/fire-statistical-reports/fire-safety-in-the-united-states',
            'https://www.nfpa.org/news-blogs-and-articles/blogs/2024/11/06/history-of-nfpa',
            'https://www.nfpa.org/about-nfpa/why-nfpa-codes-and-standards-matter',
            'https://www.nfpa.org/for-professionals/codes-and-standards/list-of-codes-and-standards#aq=%40culture%3D&cq=%40tagtype%3D%3D(%22Standards%20Development%20Process%22)%20%20&numberOfResults=12&sortCriteria=%40computedproductid%20ascending%2C%40productid%20ascending',
            'https://www.nfpa.org/education-and-research/building-and-life-safety',
            'https://www.nfpa.org/news-blogs-and-articles/NFPA-Blogs#aq=%40culture%3D%22en%22&cq=%40tagtype%3D%3D(%22Blogs%22)%20%20&numberOfResults=12&sortCriteria=%40publicationdate%20descending',
            'https://www.nfpa.org/news-blogs-and-articles/Podcasts#aq=%40culture%3D%22en%22&cq=%40tagtype%3D%3D(%22Podcasts%22)%20%20&numberOfResults=12&sortCriteria=%40publicationdate%20descending',
            'https://www.nfpa.org/about-nfpa/press-room/news-releases',
            'https://www.nfpa.org/news-blogs-and-articles/nfpa-journal/Journal-articles',
            'https://www.nfpa.org/news-blogs-and-articles/Survivor-Stories',
            'https://www.nfpa.org/news-blogs-and-articles/nfpa-journal',
            'https://www.nfpa.org/news-blogs-and-articles',
            'https://www.nfpa.org/events/fire-prevention-week',

        ]
        if DEBUG:
            start_urls = [
                start_urls[-1]
            ]
        self.start_urls.extend(start_urls)
    
    def start_requests(self):
        """重写起始请求方法，直接使用 Selenium 渲染"""
        for url in self.start_urls:
            self.logger.info(f"使用 Selenium 渲染起始页面: {url}")
            
            # 使用 Selenium 渲染页面
            html = self.selenium_renderer.render_page(url, wait_time=5)
            if html:
                # 创建已渲染的响应对象
                rendered_response = self.selenium_renderer.create_response(url, html)

                if rendered_response:
                    # 为响应对象添加必要的属性
                    rendered_response._selenium_rendered = True
                    # 直接调用 parse 方法处理渲染后的内容
                    yield from self.parse(rendered_response)
                else:
                    self.logger.error("Selenium 渲染失败，无法创建响应对象")
            else:
                self.logger.error("Selenium 渲染失败，返回空 HTML")
    
    def parse(self, response):
        """主解析方法 - 根据页面类型选择合适的解析器"""
        try:
            # 检查是否需要 Selenium 渲染
            if response.meta.get('use_selenium') and not hasattr(response, '_selenium_rendered'):
                self.logger.info(f"使用 Selenium 渲染页面: {response.url}")
                
                # 使用 Selenium 渲染页面
                html = self.selenium_renderer.render_page(response.url, wait_time=5)
                if html:
                    # 创建已渲染的响应对象，传递 meta
                    rendered_response = self.selenium_renderer.create_response(response.url, html, response.meta)
                    if rendered_response:
                        # 标记为已渲染
                        rendered_response._selenium_rendered = True
                        # 递归调用 parse 方法处理渲染后的内容
                        yield from self.parse(rendered_response)
                        return
                    else:
                        self.logger.error("Selenium 渲染失败，无法创建响应对象")
                else:
                    self.logger.error("Selenium 渲染失败，返回空 HTML")
                    return
            
            # 遍历所有解析器，找到能处理该响应的解析器
            for parser in self.parsers:
                can_handle = parser.can_handle(response)
                if can_handle:
                    self.logger.info(f"使用解析器: {parser.__class__.__name__}")
                    yield from parser.parse(response)
                    return
            
        except Exception as e:
            self.logger.error(f"NFPA parse方法异常: {e}")
            import traceback
            self.logger.error(f"异常详情: {traceback.format_exc()}")
    
    def closed(self, reason):
        """爬虫关闭时清理资源"""
        self.logger.info("NFPA 爬虫关闭，清理 Selenium 资源")
        close_selenium_renderer()
    
    def add_parser(self, parser_class):
        """动态添加解析器"""
        parser = parser_class(self)
        self.parsers.append(parser)
    
    def remove_parser(self, parser_class):
        """移除解析器"""
        self.parsers = [p for p in self.parsers if not isinstance(p, parser_class)] 